﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Net;
using System.Collections;
using System.Text.RegularExpressions;
using System.Threading;

namespace WebC
{
    internal class Crawler
    {
        public Hashtable urls = new Hashtable();
        public int count = 0;
        public Hashtable exMessage = new Hashtable();
        public string startUrl;

        //public string[] crawlingMessage;

        //public void crawStart(string[] args)
        //{
        //    Crawler myCrawler = new Crawler();
        //    string startUrl = "http://www.cnblogs.com/dstang2000/";
        //    if (args.Length > 0) startUrl = args[0];
        //    myCrawler.urls.Add(startUrl, false);//加入初始页面
        //    new Thread(myCrawler.Crawl).Start();
        //}
        public void Crawl()
        {
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)//找到一个还没有下载过的连接
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null || count > 0) break;         
                string html = DownLoad(current);//下载
                Regex ishtml = new Regex(@"^<!DOCTYPE html>");
                urls[current] = true;
                count++;
                if(ishtml.IsMatch(html))
                    Parse(html);
            }
        }

        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);

                string fileName = count.ToString();
                File.WriteAllText(fileName, html);
                return html;
            }
            catch (Exception ex)
            {
                exMessage.Add( url+" : "+ex.Message,false );
                return "";
            }
        }

        public void Parse(string html)
        {
            //string strRef =  @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";//找出html文本中的超链接
            string strRef = @"(http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                //strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"','#','>');//超链接转化为网页地址
                strRef = match.Value;
                if (strRef.Length == 0) continue;
                if (urls[strRef] == null) 
                    urls.Add(strRef, false);
            }
        }
    }
}
